package com.uusee.crawler.job;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

import net.sf.json.JSONObject;

import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.uusee.crawler.dbwriter.UgcDbWriter;
import com.uusee.crawler.fetcher.FetchHTTP;
import com.uusee.crawler.framework.CrawlController;
import com.uusee.crawler.framework.Job;
import com.uusee.crawler.model.CrawlStatusCodes;
import com.uusee.crawler.model.CrawlURI;
import com.uusee.crawler.pageprocessor.video.PlayPageProcessor;
import com.uusee.crawler.pageprocessor.video.sina.SinaUserVideoInterfacePageProcessor;
import com.uusee.crawler.xmlwriter.UgcXmlWriter;
import com.uusee.framework.bo.UniversalBo;
import com.uusee.framework.util.query.CriteriaInfo;
import com.uusee.shipshape.gt.model.UploadUser;
import com.uusee.util.StringUtils;

public class SinaUserVideoCrawlJob {
	private static final int PAGE_SIZE = 10;
	
	private Log log = LogFactory.getLog(SinaUserVideoCrawlJob.class);
	private String videoSite = "v.sina.com.cn";
	private int start = 1260000000;
	private int end =   1300000000;
	
	private FetchHTTP fetch = new FetchHTTP();
	private SinaUserVideoInterfacePageProcessor userVideoPageProcessor = new SinaUserVideoInterfacePageProcessor();
	private UgcXmlWriter xmlWriter = new UgcXmlWriter();
	private UgcDbWriter dbWriter = new UgcDbWriter();
	
	private UniversalBo universalBo;
	
	private Job ugcCrawlJob = new SinaUgcCrawlJob();
	private CrawlController crawlController = new CrawlController();
	
	public void doExecute() {
		try {
			StopWatch stopWatch = new StopWatch();
			stopWatch.start();
			int totalCrawlUser = 0;
			int totalPage = 0;
			
			//抓取数据库用户
			CriteriaInfo ci = new CriteriaInfo();
			ci.eq("sourceSite", "v.sina.com.cn");
			List<UploadUser> uploadUserList = universalBo.getEntitiesByCriteriaInfo(UploadUser.class, ci);
			for (UploadUser u:uploadUserList) {
				String userId = u.getUserId();
				try {
					int maxPage = getMaxPage(userId);
					if (maxPage>0) {
						totalCrawlUser++;
						totalPage=totalPage+maxPage;
						log.info(videoSite+"-抓取视频(用户:"+userId+";视频页数："+maxPage+")");
						
						for (int page = 1;page<=maxPage;page++) {
							CrawlURI crawURI = new CrawlURI();
							crawURI.setSourceSite(videoSite);
							crawURI.setCrawlUrl("http://you.video.sina.com.cn/api/videoList.php?uid="+userId+"&channelId=&page="+page+"&varname=");
							crawURI.setModel(new ArrayList<CrawlURI>());
							crawl(crawURI);
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
				if (totalCrawlUser%100==0) {
					log.info(videoSite+"-抓取用户数:"+totalCrawlUser+";抓取视频页数："+totalPage+";用时:"+stopWatch.getTime()/1000+"秒");
				}
			}
			
			//抓取用户
			for (int userId = start; userId < end; userId++) {
				try {
					int maxPage = getMaxPage(userId+"");
					if (maxPage>0) {
						totalCrawlUser++;
						totalPage=totalPage+maxPage;
						log.info(videoSite+"-抓取视频(用户:"+userId+";视频页数："+maxPage+")");
						
						for (int page = 1;page<=maxPage;page++) {
							CrawlURI crawURI = new CrawlURI();
							crawURI.setSourceSite(videoSite);
							crawURI.setCrawlUrl("http://you.video.sina.com.cn/api/videoList.php?uid="+userId+"&channelId=&page="+page+"&varname=");
							crawURI.setModel(new ArrayList<CrawlURI>());
							crawl(crawURI);
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
				if (userId%100==0) {
					log.info(videoSite+"-抓取用户数:"+(userId-start)+";抓取视频页数："+totalPage+";用时:"+stopWatch.getTime()/1000+"秒");
				}
			}
			stopWatch.stop();
			log.info("抓取结束,用时：" + stopWatch.getTime() / 1000 + "秒。");
		} catch (Exception e) {
			log.error(e);
		}
	}
	
	private int getMaxPage(String userId) {
		int maxPage = 0;
		try {
			CrawlURI crawlURI = new CrawlURI();
			crawlURI.setSourceSite(videoSite);
			crawlURI.setCrawlUrl("http://you.video.sina.com.cn/api/videoList.php?uid="+userId+"&channelId=&page=1&varname=");
			fetch.process(crawlURI);
			int crawlStatus = crawlURI.getCrawlStatus();
			if (crawlStatus == CrawlStatusCodes.FETCH_SUCCESS) {
				JSONObject jsonObject = JSONObject.fromObject( crawlURI.getCrawlResult() ); 
				int videoCount = jsonObject.getJSONObject("data").getInt("count");
				if (videoCount != 0 ) {
					maxPage = videoCount/PAGE_SIZE+1;
				}
			}
		}catch(Exception e){
		}
		return maxPage;
	}

	public void crawl(CrawlURI crawURI) {
		try {
			fetch.process(crawURI);
			userVideoPageProcessor.process(crawURI);
			if (crawURI.getCrawlStatus() == CrawlStatusCodes.PAGE_PROCESS_SUCCESS) {
				List<CrawlURI> playCrawURIList = (List<CrawlURI>) crawURI.getModel();
				for (CrawlURI playCrawURI:playCrawURIList) {
					//dbWriter.process(playCrawURI);
					playCrawURI.setCrawlStatus(CrawlStatusCodes.DEFAULT_STATUS);
					//ugcCrawlJob.crawl(playCrawURI);
					crawlController.assignTask(ugcCrawlJob, playCrawURI);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public CrawlController getCrawlController() {
		return crawlController;
	}

	public void setCrawlController(CrawlController crawlController) {
		this.crawlController = crawlController;
	}

	public void setStart(int start) {
		this.start = start;
	}

	public void setEnd(int end) {
		this.end = end;
	}

	public void setFetch(FetchHTTP fetch) {
		this.fetch = fetch;
	}

	public void setUserVideoPageProcessor(SinaUserVideoInterfacePageProcessor userVideoPageProcessor) {
		this.userVideoPageProcessor = userVideoPageProcessor;
	}

	public void setXmlWriter(UgcXmlWriter xmlWriter) {
		this.xmlWriter = xmlWriter;
	}

	public void setDbWriter(UgcDbWriter dbWriter) {
		this.dbWriter = dbWriter;
	}

	public void setUniversalBo(UniversalBo universalBo) {
		this.universalBo = universalBo;
	}

	public void setUgcCrawlJob(Job ugcCrawlJob) {
		this.ugcCrawlJob = ugcCrawlJob;
	}

	public static void main(String[] args) {
		try {
			ApplicationContext acx = new ClassPathXmlApplicationContext("spring/applicationContext-framework.xml");
			SinaUserVideoCrawlJob job = new SinaUserVideoCrawlJob();
			UniversalBo universalBo = (UniversalBo)acx.getBean("universalBo");
			job.dbWriter.setUniversalBo(universalBo);
			job.universalBo = universalBo;
			
			SinaUgcCrawlJob ugcCrawlJob = new SinaUgcCrawlJob();
			InputStream in = ClassLoader.getSystemResourceAsStream("regex/v.sina.com.cn/playPage.properties"); 
			Properties processPageRegex = new Properties();
			processPageRegex.load(in); 
			PlayPageProcessor playPageProcessor = new PlayPageProcessor();
			playPageProcessor.setPageProcessRegex(processPageRegex);
			ugcCrawlJob.setFetch(job.fetch);
			ugcCrawlJob.setDbWriter(job.dbWriter);
			ugcCrawlJob.setPageProcessor(playPageProcessor);
			
			job.ugcCrawlJob = ugcCrawlJob;
			
			job.doExecute();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}
